# -*- coding: utf-8 -*-
import scrapy
from kuaidaili.items import KuaidailiItem
import time

class KuaidailispiderSpider(scrapy.Spider):
    name = 'kuaidailispider'
    allowed_domains = ['www.kuaidaili.com']
    # 基础 url
    url = 'https://www.kuaidaili.com/free/intr/'
    # 起始页面
    page = 1
    # 起始 url
    start_urls = ['https://www.kuaidaili.com/free/intr/1']

    def parse(self, response):
        print('开始爬取...')
        ip_list = response.xpath('//tbody/tr/td[1]/text()').extract()
        port_list = response.xpath('//tbody/tr/td[2]/text()').extract()
        for i in range(len(ip_list)):
            item = KuaidailiItem()
            item['ip'] = ip_list[i]
            item['port'] = port_list[i]
            yield item
        time.sleep(1)
        self.page += 1
        if self.page <= 10:
            url = self.url + str(self.page)
            print('kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk')
            print(url)
            yield scrapy.Request(url=url,callback=self.parse)